{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "    http://www.apache.org/licenses/LICENSE-2.0\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1,2,3\"\n",
    "start = time.time()\n",
    "very_start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.14.0a+3537.g4cafc98.dirty'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import pandas as pd, \n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "#pd.set_option('display.max_columns', 500)\n",
    "#pd.set_option('display.max_rows', 500)\n",
    "import cudf, cupy, time, rmm\n",
    "cudf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask as dask, dask_cudf\n",
    "from dask.distributed import Client, wait\n",
    "from dask_cuda import LocalCUDACluster\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169.254.8.236\n"
     ]
    }
   ],
   "source": [
    "cmd = \"hostname --all-ip-addresses\"\n",
    "process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)\n",
    "output, error = process.communicate()\n",
    "IPADDR = str(output.decode()).split()[0]\n",
    "print(IPADDR)\n",
    "# this is not the correct ip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>tcp://10.2.48.253:41663</li>\n",
       "  <li><b>Dashboard: </b><a href='http://10.2.48.253:8787/status' target='_blank'>http://10.2.48.253:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>4</li>\n",
       "  <li><b>Memory: </b>270.39 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: 'tcp://10.2.48.253:41663' processes=4 threads=4, memory=270.39 GB>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster = LocalCUDACluster(ip='10.2.48.253')\n",
    "client = Client(cluster)\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#client.run(cudf.set_allocator, pool=True, initial_pool_size=int(2**35)-int(2**32))\n",
    "#client.run(cupy.cuda.set_allocator, rmm.rmm_cupy_allocator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 853 ms, sys: 427 ms, total: 1.28 s\n",
      "Wall time: 1.27 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "path = '/raid/data/recsys/train_split'\n",
    "train = dask_cudf.read_parquet(f'{path}/train-preproc-fold-*.parquet')#,dtypes=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 63.9 ms, sys: 10.4 ms, total: 74.3 ms\n",
      "Wall time: 66.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# DROP UNUSED COLUMNS\n",
    "cols_drop = ['links','hashtags0', 'hashtags1', 'fold']\n",
    "train = train.drop(cols_drop,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-beb031f7-2d8c-4923-9dc0-7b2c47be50d5'), 25)\n",
      "CPU times: user 16.5 ms, sys: 6.62 ms, total: 23.2 ms\n",
      "Wall time: 21.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 430 ms, sys: 68.2 ms, total: 499 ms\n",
      "Wall time: 8.89 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>b_account_creation</th>\n",
       "      <th>b_follows_a</th>\n",
       "      <th>reply</th>\n",
       "      <th>retweet</th>\n",
       "      <th>retweet_comment</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>92560312</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>2020-02-09 09:26:50</td>\n",
       "      <td>0</td>\n",
       "      <td>14326</td>\n",
       "      <td>408</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-03-13 13:47:49</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55533709</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>2020-02-09 18:41:35</td>\n",
       "      <td>10</td>\n",
       "      <td>237126</td>\n",
       "      <td>1193</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>2011-09-05 16:42:09</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37019981</th>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2020-02-09 01:13:28</td>\n",
       "      <td>19</td>\n",
       "      <td>23079</td>\n",
       "      <td>1803</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>2016-05-19 02:19:01</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74047136</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>2020-02-07 12:15:20</td>\n",
       "      <td>29</td>\n",
       "      <td>769176</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>2019-09-10 09:17:08</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-02-07 12:36:47</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74047139</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>2020-02-08 14:14:39</td>\n",
       "      <td>35</td>\n",
       "      <td>73952</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>2019-12-11 15:38:45</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-02-09 13:33:47</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          tweet_id  media  domains  tweet_type  language           timestamp  \\\n",
       "92560312         0      7        0           1        54 2020-02-09 09:26:50   \n",
       "55533709        10      0        0           1        54 2020-02-09 18:41:35   \n",
       "37019981        20      5        0           1         3 2020-02-09 01:13:28   \n",
       "74047136        30      0        5           2        11 2020-02-07 12:15:20   \n",
       "74047139        40      0        0           2         6 2020-02-08 14:14:39   \n",
       "\n",
       "          a_user_id  a_follower_count  a_following_count  a_is_verified  ...  \\\n",
       "92560312          0             14326                408          False  ...   \n",
       "55533709         10            237126               1193           True  ...   \n",
       "37019981         19             23079               1803          False  ...   \n",
       "74047136         29            769176                190          False  ...   \n",
       "74047139         35             73952                 13          False  ...   \n",
       "\n",
       "          b_account_creation b_follows_a  reply  retweet  retweet_comment  \\\n",
       "92560312 2018-03-13 13:47:49        True      0        0                0   \n",
       "55533709 2011-09-05 16:42:09       False      0        0                0   \n",
       "37019981 2016-05-19 02:19:01       False      0        0                0   \n",
       "74047136 2019-09-10 09:17:08       False      0        0                0   \n",
       "74047139 2019-12-11 15:38:45       False      0        0                0   \n",
       "\n",
       "          like         engage_time len_domains  len_hashtags  len_links  \n",
       "92560312     0 1970-01-01 00:00:00           0             0          0  \n",
       "55533709     0 1970-01-01 00:00:00           0             0          0  \n",
       "37019981     0 1970-01-01 00:00:00           0             0          0  \n",
       "74047136     1 2020-02-07 12:36:47           0             0          0  \n",
       "74047139     1 2020-02-09 13:33:47           0             0          0  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = train.repartition(npartitions=4)\n",
    "train, = dask.persist(train)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 29629913\n",
      "1 44309248\n",
      "2 29638018\n",
      "3 44498059\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(i, len(train.partitions[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = ['reply', 'retweet', 'retweet_comment', 'like']\n",
    "for col in train.columns:\n",
    "    if col in label_names:\n",
    "        train[col] = train[col].astype('float32')\n",
    "    elif train[col].dtype=='int64':\n",
    "        train[col] = train[col].astype('int32')\n",
    "    elif train[col].dtype=='int16':\n",
    "        train[col] = train[col].astype('int8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 61.9 ms, sys: 268 µs, total: 62.2 ms\n",
      "Wall time: 58.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train = train.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Delayed('int-a8eef28e-0fe4-44f0-8080-36a07abe1271'), 25)\n",
      "CPU times: user 54.9 ms, sys: 677 µs, total: 55.5 ms\n",
      "Wall time: 53.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 66.5 ms, sys: 3.8 ms, total: 70.3 ms\n",
      "Wall time: 65.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# TIME FEATURES\n",
    "# RAPIDS does this 5x faster than Pandas CPU\n",
    "# If we didn't need to copy CPU to GPU to CPU, then 1300x faster!\n",
    "def split_time(df):\n",
    "    #gf = cudf.from_pandas(df[['timestamp']])\n",
    "    df['dt_dow']  = df['timestamp'].dt.weekday#.to_array() \n",
    "    df['dt_hour'] = df['timestamp'].dt.hour#.to_array()\n",
    "    df['dt_minute'] = df['timestamp'].dt.minute#.to_array()\n",
    "    df['dt_second'] = df['timestamp'].dt.second#.to_array()\n",
    "    return df\n",
    "\n",
    "train = split_time(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>engage_time</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 09:26:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 18:41:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1970-01-01 00:00:00</td>\n",
       "      <td>2020-02-09 01:13:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-02-07 12:36:47</td>\n",
       "      <td>2020-02-07 12:15:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-02-09 13:33:47</td>\n",
       "      <td>2020-02-08 14:14:39</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          engage_time           timestamp\n",
       "0 1970-01-01 00:00:00 2020-02-09 09:26:50\n",
       "1 1970-01-01 00:00:00 2020-02-09 18:41:35\n",
       "2 1970-01-01 00:00:00 2020-02-09 01:13:28\n",
       "3 2020-02-07 12:36:47 2020-02-07 12:15:20\n",
       "4 2020-02-09 13:33:47 2020-02-08 14:14:39"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()[['engage_time','timestamp']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('<M8[ns]')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.timestamp.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47 ms, sys: 646 µs, total: 47.7 ms\n",
      "Wall time: 44.7 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# ELAPSED TIME\n",
    "for col in ['engage_time','timestamp']:\n",
    "    train[col] = train[col].astype('int64')/1e3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-e5dcc896-4aaf-4335-bbd4-139c9f5ecaa1'), 29)\n",
      "CPU times: user 19.2 ms, sys: 3.09 ms, total: 22.3 ms\n",
      "Wall time: 20.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>engage_time</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581240e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581274e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.581211e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.581079e+09</td>\n",
       "      <td>1.581078e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.581255e+09</td>\n",
       "      <td>1.581171e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    engage_time     timestamp\n",
       "0  0.000000e+00  1.581240e+09\n",
       "1  0.000000e+00  1.581274e+09\n",
       "2  0.000000e+00  1.581211e+09\n",
       "3  1.581079e+09  1.581078e+09\n",
       "4  1.581255e+09  1.581171e+09"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()[['engage_time','timestamp']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_nan(ds):\n",
    "    mask = ds == 0\n",
    "    ds.loc[mask] = np.nan\n",
    "    return ds.nans_to_nulls()\n",
    "train['engage_time'] = train['engage_time'].map_partitions(set_nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['elapsed_time'] = train['engage_time'] - train['timestamp']\n",
    "train['elapsed_time'] = train.elapsed_time.astype('float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0 603956.0\n",
      "15581.699535245267\n"
     ]
    }
   ],
   "source": [
    "print(train['elapsed_time'].min().compute(),train['elapsed_time'].max().compute())\n",
    "print(train['elapsed_time'].mean().compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-7ffc6228-34d5-4292-9d8f-ac35561aa868'), 30)\n",
      "CPU times: user 32.4 ms, sys: 0 ns, total: 32.4 ms\n",
      "Wall time: 30.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train, = dask.persist(train)\n",
    "print(type(train), train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>dt_minute</th>\n",
       "      <th>dt_second</th>\n",
       "      <th>elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1.581240e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>14326</td>\n",
       "      <td>408</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>26</td>\n",
       "      <td>50</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>1.581274e+09</td>\n",
       "      <td>10</td>\n",
       "      <td>237126</td>\n",
       "      <td>1193</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>41</td>\n",
       "      <td>35</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.581211e+09</td>\n",
       "      <td>19</td>\n",
       "      <td>23079</td>\n",
       "      <td>1803</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>28</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1.581078e+09</td>\n",
       "      <td>29</td>\n",
       "      <td>769176</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581079007e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>1287.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1.581171e+09</td>\n",
       "      <td>35</td>\n",
       "      <td>73952</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.581255227e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>39</td>\n",
       "      <td>83948.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id  \\\n",
       "0         0      7        0           1        54  1.581240e+09          0   \n",
       "1        10      0        0           1        54  1.581274e+09         10   \n",
       "2        20      5        0           1         3  1.581211e+09         19   \n",
       "3        30      0        5           2        11  1.581078e+09         29   \n",
       "4        40      0        0           2         6  1.581171e+09         35   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  ... like  \\\n",
       "0             14326                408          False  ...  0.0   \n",
       "1            237126               1193           True  ...  0.0   \n",
       "2             23079               1803          False  ...  0.0   \n",
       "3            769176                190          False  ...  1.0   \n",
       "4             73952                 13          False  ...  1.0   \n",
       "\n",
       "       engage_time len_domains  len_hashtags  len_links  dt_dow  dt_hour  \\\n",
       "0             null           0             0          0       6        9   \n",
       "1             null           0             0          0       6       18   \n",
       "2             null           0             0          0       6        1   \n",
       "3  1.581079007e+09           0             0          0       4       12   \n",
       "4  1.581255227e+09           0             0          0       5       14   \n",
       "\n",
       "   dt_minute  dt_second  elapsed_time  \n",
       "0         26         50          null  \n",
       "1         41         35          null  \n",
       "2         13         28          null  \n",
       "3         15         20        1287.0  \n",
       "4         14         39       83948.0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 78.1 ms, sys: 3.31 ms, total: 81.4 ms\n",
      "Wall time: 75.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# TRAIN FIRST 5 DAYS. VALIDATE LAST 2 DAYS\n",
    "VALID_DOW = [1, 2]# order is [3, 4, 5, 6, 0, 1, 2]\n",
    "valid = train[train['dt_dow'].isin(VALID_DOW)].reset_index(drop=True)\n",
    "train = train[~train['dt_dow'].isin(VALID_DOW)].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask_cudf.core.DataFrame'> (Delayed('int-1450cecc-05d4-419c-9636-33f44bd89aee'), 30) (Delayed('int-1523bcda-1447-41d4-8b2e-e11a2aa98159'), 30)\n",
      "CPU times: user 27.8 ms, sys: 245 µs, total: 28 ms\n",
      "Wall time: 24.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "print(type(train), train.shape, valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.11 s, sys: 72.2 ms, total: 1.18 s\n",
      "Wall time: 6.15 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>like</th>\n",
       "      <th>engage_time</th>\n",
       "      <th>len_domains</th>\n",
       "      <th>len_hashtags</th>\n",
       "      <th>len_links</th>\n",
       "      <th>dt_dow</th>\n",
       "      <th>dt_hour</th>\n",
       "      <th>dt_minute</th>\n",
       "      <th>dt_second</th>\n",
       "      <th>elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>164980</td>\n",
       "      <td>5</td>\n",
       "      <td>6625</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>112668</td>\n",
       "      <td>22984</td>\n",
       "      <td>6033</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1173270</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>672136</td>\n",
       "      <td>5341</td>\n",
       "      <td>840</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3240270</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>64906</td>\n",
       "      <td>256871</td>\n",
       "      <td>8</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2468740</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>1278002</td>\n",
       "      <td>16289</td>\n",
       "      <td>376</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3653210</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1.580947e+09</td>\n",
       "      <td>98180</td>\n",
       "      <td>9841</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id  \\\n",
       "0    164980      5     6625           2         3  1.580947e+09     112668   \n",
       "1   1173270      0        4           2        54  1.580947e+09     672136   \n",
       "2   3240270      9        0           2        11  1.580947e+09      64906   \n",
       "3   2468740      5        4           2        54  1.580947e+09    1278002   \n",
       "4   3653210      0        0           2        11  1.580947e+09      98180   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  ... like  engage_time  \\\n",
       "0             22984               6033           True  ...  0.0         null   \n",
       "1              5341                840          False  ...  0.0         null   \n",
       "2            256871                  8           True  ...  0.0         null   \n",
       "3             16289                376          False  ...  0.0         null   \n",
       "4              9841                  6           True  ...  0.0         null   \n",
       "\n",
       "  len_domains  len_hashtags  len_links  dt_dow  dt_hour  dt_minute  dt_second  \\\n",
       "0           0             0          0       3        0          0          0   \n",
       "1           0             0          0       3        0          0          0   \n",
       "2           0             2          0       3        0          0          0   \n",
       "3           0             0          0       3        0          0          0   \n",
       "4           0             5          0       3        0          0          0   \n",
       "\n",
       "   elapsed_time  \n",
       "0          null  \n",
       "1          null  \n",
       "2          null  \n",
       "3          null  \n",
       "4          null  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = train.sort_values('timestamp').reset_index(drop=True)\n",
    "valid = valid.sort_values('timestamp').reset_index(drop=True)\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 26885321\n",
      "1 26832264\n",
      "2 26822864\n",
      "3 26696073\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(i,len(train.partitions[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 10236930\n",
      "1 10267563\n",
      "2 10191750\n",
      "3 10142473\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(i,len(valid.partitions[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MTE_one_shot:\n",
    "    \n",
    "    def __init__(self, folds, smooth, seed=42, mode='gpu'):\n",
    "        self.folds = folds\n",
    "        self.seed = seed\n",
    "        self.smooth = smooth\n",
    "        if mode=='gpu':\n",
    "            self.np = cupy\n",
    "            self.df = cudf\n",
    "        else:\n",
    "            self.np = np\n",
    "            self.df = pd\n",
    "        self.mode = mode\n",
    "        \n",
    "    def fit_transform(self, train, x_col, y_col, y_mean=None, out_col = None, out_dtype=None):\n",
    "        \n",
    "        self.y_col = y_col\n",
    "        self.np.random.seed(self.seed)\n",
    "        \n",
    "        if 'fold' not in train.columns:\n",
    "            fsize = len(train)//self.folds\n",
    "            if isinstance(train,dask_cudf.core.DataFrame):\n",
    "                #train['fold'] = train.map_partitions(lambda cudf_df: cudf_df.index%self.folds)\n",
    "                train['fold'] = 1\n",
    "                train['fold'] = train['fold'].cumsum()\n",
    "                train['fold'] = train['fold']//fsize\n",
    "                train['fold'] = train['fold']%self.folds\n",
    "            else:\n",
    "                #train['fold'] = self.np.random.randint(0,self.folds,len(train))\n",
    "                train['fold'] = (train.index.values//fsize)%self.folds\n",
    "        \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'TE_{tag}_{self.y_col}'\n",
    "        \n",
    "        if y_mean is None:\n",
    "            y_mean = train[y_col].mean()#.compute().astype('float32')\n",
    "        self.mean = y_mean\n",
    "        \n",
    "        cols = ['fold',x_col] if isinstance(x_col,str) else ['fold']+x_col\n",
    "        \n",
    "        agg_each_fold = train.groupby(cols).agg({y_col:['count','sum']}).reset_index()\n",
    "        agg_each_fold.columns = cols + ['count_y','sum_y']\n",
    "        \n",
    "        agg_all = agg_each_fold.groupby(x_col).agg({'count_y':'sum','sum_y':'sum'}).reset_index()\n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all.columns = cols + ['count_y_all','sum_y_all']\n",
    "        \n",
    "        agg_each_fold = agg_each_fold.merge(agg_all,on=x_col,how='left')\n",
    "        agg_each_fold['count_y_all'] = agg_each_fold['count_y_all'] - agg_each_fold['count_y']\n",
    "        agg_each_fold['sum_y_all'] = agg_each_fold['sum_y_all'] - agg_each_fold['sum_y']\n",
    "        agg_each_fold[out_col] = (agg_each_fold['sum_y_all']+self.smooth*self.mean)/(agg_each_fold['count_y_all']+self.smooth)\n",
    "        agg_each_fold = agg_each_fold.drop(['count_y_all','count_y','sum_y_all','sum_y'],axis=1)\n",
    "        \n",
    "        agg_all[out_col] = (agg_all['sum_y_all']+self.smooth*self.mean)/(agg_all['count_y_all']+self.smooth)\n",
    "        agg_all = agg_all.drop(['count_y_all','sum_y_all'],axis=1)\n",
    "        self.agg_all = agg_all\n",
    "        \n",
    "        train.columns\n",
    "        cols = ['fold',x_col] if isinstance(x_col,str) else ['fold']+x_col\n",
    "        train = train.merge(agg_each_fold,on=cols,how='left')\n",
    "        del agg_each_fold\n",
    "        #self.agg_each_fold = agg_each_fold\n",
    "        if self.mode=='gpu':\n",
    "            if isinstance(train,dask_cudf.core.DataFrame):\n",
    "                train[out_col] = train.map_partitions(lambda cudf_df: cudf_df[out_col].nans_to_nulls())\n",
    "            else:\n",
    "                train[out_col] = train[out_col].nans_to_nulls()\n",
    "        train[out_col] = train[out_col].fillna(self.mean)\n",
    "        \n",
    "        if out_dtype is not None:\n",
    "            train[out_col] = train[out_col].astype(out_dtype)\n",
    "        return train\n",
    "    \n",
    "    def transform(self, test, x_col, out_col = None, out_dtype=None):\n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'TE_{tag}_{self.y_col}'\n",
    "        test = test.merge(self.agg_all,on=x_col,how='left')\n",
    "        test[out_col] = test[out_col].fillna(self.mean)\n",
    "        if out_dtype is not None:\n",
    "            test[out_col] = test[out_col].astype(out_dtype)\n",
    "        return test\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18 s, sys: 1.26 s, total: 19.3 s\n",
      "Wall time: 1min 29s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "start = time.time()\n",
    "for t in ['reply', 'retweet', 'retweet_comment', 'like']:\n",
    "    start = time.time()\n",
    "    for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "        out_col = f'TE_{c}_{t}'\n",
    "        encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "        train = encoder.fit_transform(train, c, t, out_col=out_col, out_dtype='float32')\n",
    "        valid = encoder.transform(valid, c, out_col=out_col, out_dtype='float32')\n",
    "        cols.append(out_col)\n",
    "        del encoder\n",
    "        train,valid = dask.persist(train,valid)\n",
    "        train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    21447306\n",
       "1    21447304\n",
       "2    21447304\n",
       "3    21447304\n",
       "4    21447304\n",
       "Name: fold, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['fold'].value_counts().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiple Column Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.61 s, sys: 96.4 ms, total: 2.71 s\n",
      "Wall time: 2.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols=[]\n",
    "c = ['domains','language','b_follows_a','tweet_type','media','a_is_verified']\n",
    "for t in ['reply', 'retweet', 'retweet_comment', 'like']:\n",
    "    out_col = f'TE_multi_{t}'\n",
    "    encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "    train = encoder.fit_transform(train, c, t, out_col=out_col, out_dtype='float32')\n",
    "    valid = encoder.transform(valid, c, out_col=out_col, out_dtype='float32')\n",
    "    cols.append(out_col)\n",
    "    del encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.58 s, sys: 130 ms, total: 1.71 s\n",
      "Wall time: 8.92 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>TE_b_user_id_retweet_comment</th>\n",
       "      <th>TE_media_like</th>\n",
       "      <th>TE_tweet_type_like</th>\n",
       "      <th>TE_language_like</th>\n",
       "      <th>TE_a_user_id_like</th>\n",
       "      <th>TE_b_user_id_like</th>\n",
       "      <th>TE_multi_reply</th>\n",
       "      <th>TE_multi_retweet</th>\n",
       "      <th>TE_multi_retweet_comment</th>\n",
       "      <th>TE_multi_like</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62920</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>781</td>\n",
       "      <td>562432</td>\n",
       "      <td>327</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007708</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.790396</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.027371</td>\n",
       "      <td>0.102288</td>\n",
       "      <td>0.007343</td>\n",
       "      <td>0.603071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62920</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>781</td>\n",
       "      <td>562432</td>\n",
       "      <td>327</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006167</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.790396</td>\n",
       "      <td>0.439130</td>\n",
       "      <td>0.027371</td>\n",
       "      <td>0.102288</td>\n",
       "      <td>0.007343</td>\n",
       "      <td>0.603071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62920</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>781</td>\n",
       "      <td>555910</td>\n",
       "      <td>325</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007341</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.790396</td>\n",
       "      <td>0.427535</td>\n",
       "      <td>0.027371</td>\n",
       "      <td>0.102288</td>\n",
       "      <td>0.007343</td>\n",
       "      <td>0.603071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>62920</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>781</td>\n",
       "      <td>562432</td>\n",
       "      <td>327</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005316</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.790396</td>\n",
       "      <td>0.482008</td>\n",
       "      <td>0.027371</td>\n",
       "      <td>0.102288</td>\n",
       "      <td>0.007343</td>\n",
       "      <td>0.603071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>62920</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>781</td>\n",
       "      <td>562432</td>\n",
       "      <td>327</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006167</td>\n",
       "      <td>0.494331</td>\n",
       "      <td>0.532514</td>\n",
       "      <td>0.449791</td>\n",
       "      <td>0.790396</td>\n",
       "      <td>0.479130</td>\n",
       "      <td>0.027371</td>\n",
       "      <td>0.102288</td>\n",
       "      <td>0.007343</td>\n",
       "      <td>0.603071</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id  \\\n",
       "0     62920      5        0           2        54  1.580948e+09        781   \n",
       "1     62920      5        0           2        54  1.580948e+09        781   \n",
       "2     62920      5        0           2        54  1.580948e+09        781   \n",
       "3     62920      5        0           2        54  1.580948e+09        781   \n",
       "4     62920      5        0           2        54  1.580948e+09        781   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  ...  \\\n",
       "0            562432                327          False  ...   \n",
       "1            562432                327          False  ...   \n",
       "2            555910                325          False  ...   \n",
       "3            562432                327          False  ...   \n",
       "4            562432                327          False  ...   \n",
       "\n",
       "  TE_b_user_id_retweet_comment  TE_media_like  TE_tweet_type_like  \\\n",
       "0                     0.007708       0.494331            0.532514   \n",
       "1                     0.006167       0.494331            0.532514   \n",
       "2                     0.007341       0.494331            0.532514   \n",
       "3                     0.005316       0.494331            0.532514   \n",
       "4                     0.006167       0.494331            0.532514   \n",
       "\n",
       "   TE_language_like  TE_a_user_id_like  TE_b_user_id_like  TE_multi_reply  \\\n",
       "0          0.449791           0.790396           0.448912        0.027371   \n",
       "1          0.449791           0.790396           0.439130        0.027371   \n",
       "2          0.449791           0.790396           0.427535        0.027371   \n",
       "3          0.449791           0.790396           0.482008        0.027371   \n",
       "4          0.449791           0.790396           0.479130        0.027371   \n",
       "\n",
       "   TE_multi_retweet  TE_multi_retweet_comment  TE_multi_like  \n",
       "0          0.102288                  0.007343       0.603071  \n",
       "1          0.102288                  0.007343       0.603071  \n",
       "2          0.102288                  0.007343       0.603071  \n",
       "3          0.102288                  0.007343       0.603071  \n",
       "4          0.102288                  0.007343       0.603071  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elapsed Time Target Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.4 s, sys: 54.2 ms, total: 1.45 s\n",
      "Wall time: 1.38 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF TE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language']:#, 'a_user_id', 'b_user_id']:\n",
    "    for t in ['elapsed_time']:\n",
    "        out_col = f'TE_{c}_{t}'\n",
    "        encoder = MTE_one_shot(folds=5,smooth=20)\n",
    "        train = encoder.fit_transform(train, c, t, out_col=out_col)\n",
    "        out_dtype='float32' #if 'user_id' in c else None\n",
    "        valid = encoder.transform(valid, c, out_col=out_col, out_dtype=out_dtype)\n",
    "        cols.append(out_col)\n",
    "        #del encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 909 ms, sys: 41.1 ms, total: 950 ms\n",
      "Wall time: 5.47 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>TE_language_like</th>\n",
       "      <th>TE_a_user_id_like</th>\n",
       "      <th>TE_b_user_id_like</th>\n",
       "      <th>TE_multi_reply</th>\n",
       "      <th>TE_multi_retweet</th>\n",
       "      <th>TE_multi_retweet_comment</th>\n",
       "      <th>TE_multi_like</th>\n",
       "      <th>TE_media_elapsed_time</th>\n",
       "      <th>TE_tweet_type_elapsed_time</th>\n",
       "      <th>TE_language_elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1518712</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>1324</td>\n",
       "      <td>6103201</td>\n",
       "      <td>375</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411003</td>\n",
       "      <td>0.458515</td>\n",
       "      <td>0.390358</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>0.031647</td>\n",
       "      <td>0.000697</td>\n",
       "      <td>0.185066</td>\n",
       "      <td>17537.0768</td>\n",
       "      <td>11601.091716</td>\n",
       "      <td>19749.286736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1518712</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>1324</td>\n",
       "      <td>6071996</td>\n",
       "      <td>376</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411003</td>\n",
       "      <td>0.458515</td>\n",
       "      <td>0.433837</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>0.031647</td>\n",
       "      <td>0.000697</td>\n",
       "      <td>0.185066</td>\n",
       "      <td>17537.0768</td>\n",
       "      <td>11601.091716</td>\n",
       "      <td>19749.286736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1518712</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>1324</td>\n",
       "      <td>6103201</td>\n",
       "      <td>375</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411003</td>\n",
       "      <td>0.458515</td>\n",
       "      <td>0.408102</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>0.031647</td>\n",
       "      <td>0.000697</td>\n",
       "      <td>0.185066</td>\n",
       "      <td>17537.0768</td>\n",
       "      <td>11601.091716</td>\n",
       "      <td>19749.286736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1518712</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>1324</td>\n",
       "      <td>6103201</td>\n",
       "      <td>375</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411003</td>\n",
       "      <td>0.458515</td>\n",
       "      <td>0.427535</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>0.031647</td>\n",
       "      <td>0.000697</td>\n",
       "      <td>0.185066</td>\n",
       "      <td>17537.0768</td>\n",
       "      <td>11601.091716</td>\n",
       "      <td>19749.286736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1518712</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>1324</td>\n",
       "      <td>6103201</td>\n",
       "      <td>375</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411003</td>\n",
       "      <td>0.458515</td>\n",
       "      <td>0.453557</td>\n",
       "      <td>0.006305</td>\n",
       "      <td>0.031647</td>\n",
       "      <td>0.000697</td>\n",
       "      <td>0.185066</td>\n",
       "      <td>17537.0768</td>\n",
       "      <td>11601.091716</td>\n",
       "      <td>19749.286736</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id  \\\n",
       "0   1518712      7        0           1         3  1.580948e+09       1324   \n",
       "1   1518712      7        0           1         3  1.580948e+09       1324   \n",
       "2   1518712      7        0           1         3  1.580948e+09       1324   \n",
       "3   1518712      7        0           1         3  1.580948e+09       1324   \n",
       "4   1518712      7        0           1         3  1.580948e+09       1324   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  ... TE_language_like  \\\n",
       "0           6103201                375           True  ...         0.411003   \n",
       "1           6071996                376           True  ...         0.411003   \n",
       "2           6103201                375           True  ...         0.411003   \n",
       "3           6103201                375           True  ...         0.411003   \n",
       "4           6103201                375           True  ...         0.411003   \n",
       "\n",
       "   TE_a_user_id_like  TE_b_user_id_like  TE_multi_reply  TE_multi_retweet  \\\n",
       "0           0.458515           0.390358        0.006305          0.031647   \n",
       "1           0.458515           0.433837        0.006305          0.031647   \n",
       "2           0.458515           0.408102        0.006305          0.031647   \n",
       "3           0.458515           0.427535        0.006305          0.031647   \n",
       "4           0.458515           0.453557        0.006305          0.031647   \n",
       "\n",
       "   TE_multi_retweet_comment  TE_multi_like  TE_media_elapsed_time  \\\n",
       "0                  0.000697       0.185066             17537.0768   \n",
       "1                  0.000697       0.185066             17537.0768   \n",
       "2                  0.000697       0.185066             17537.0768   \n",
       "3                  0.000697       0.185066             17537.0768   \n",
       "4                  0.000697       0.185066             17537.0768   \n",
       "\n",
       "   TE_tweet_type_elapsed_time  TE_language_elapsed_time  \n",
       "0                11601.091716              19749.286736  \n",
       "1                11601.091716              19749.286736  \n",
       "2                11601.091716              19749.286736  \n",
       "3                11601.091716              19749.286736  \n",
       "4                11601.091716              19749.286736  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>TE_language_like</th>\n",
       "      <th>TE_a_user_id_like</th>\n",
       "      <th>TE_b_user_id_like</th>\n",
       "      <th>TE_multi_reply</th>\n",
       "      <th>TE_multi_retweet</th>\n",
       "      <th>TE_multi_retweet_comment</th>\n",
       "      <th>TE_multi_like</th>\n",
       "      <th>TE_media_elapsed_time</th>\n",
       "      <th>TE_tweet_type_elapsed_time</th>\n",
       "      <th>TE_language_elapsed_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>1.581379e+09</td>\n",
       "      <td>2824</td>\n",
       "      <td>31861000</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.635704</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>1.581379e+09</td>\n",
       "      <td>2824</td>\n",
       "      <td>31861000</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.480676</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>1.581379e+09</td>\n",
       "      <td>2824</td>\n",
       "      <td>31852634</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.479130</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>1.581379e+09</td>\n",
       "      <td>2824</td>\n",
       "      <td>31861000</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.587595</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>61737</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>1.581379e+09</td>\n",
       "      <td>2824</td>\n",
       "      <td>31852634</td>\n",
       "      <td>80</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423782</td>\n",
       "      <td>0.414891</td>\n",
       "      <td>0.477315</td>\n",
       "      <td>0.016941</td>\n",
       "      <td>0.036669</td>\n",
       "      <td>0.004081</td>\n",
       "      <td>0.472134</td>\n",
       "      <td>18000.140625</td>\n",
       "      <td>19014.623047</td>\n",
       "      <td>17206.958984</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id  \\\n",
       "0     61737      5        0           2        50  1.581379e+09       2824   \n",
       "1     61737      5        0           2        50  1.581379e+09       2824   \n",
       "2     61737      5        0           2        50  1.581379e+09       2824   \n",
       "3     61737      5        0           2        50  1.581379e+09       2824   \n",
       "4     61737      5        0           2        50  1.581379e+09       2824   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  ... TE_language_like  \\\n",
       "0          31861000                 80           True  ...         0.423782   \n",
       "1          31861000                 80           True  ...         0.423782   \n",
       "2          31852634                 80           True  ...         0.423782   \n",
       "3          31861000                 80           True  ...         0.423782   \n",
       "4          31852634                 80           True  ...         0.423782   \n",
       "\n",
       "   TE_a_user_id_like  TE_b_user_id_like  TE_multi_reply  TE_multi_retweet  \\\n",
       "0           0.414891           0.635704        0.016941          0.036669   \n",
       "1           0.414891           0.480676        0.016941          0.036669   \n",
       "2           0.414891           0.479130        0.016941          0.036669   \n",
       "3           0.414891           0.587595        0.016941          0.036669   \n",
       "4           0.414891           0.477315        0.016941          0.036669   \n",
       "\n",
       "   TE_multi_retweet_comment  TE_multi_like  TE_media_elapsed_time  \\\n",
       "0                  0.004081       0.472134           18000.140625   \n",
       "1                  0.004081       0.472134           18000.140625   \n",
       "2                  0.004081       0.472134           18000.140625   \n",
       "3                  0.004081       0.472134           18000.140625   \n",
       "4                  0.004081       0.472134           18000.140625   \n",
       "\n",
       "   TE_tweet_type_elapsed_time  TE_language_elapsed_time  \n",
       "0                19014.623047              17206.958984  \n",
       "1                19014.623047              17206.958984  \n",
       "2                19014.623047              17206.958984  \n",
       "3                19014.623047              17206.958984  \n",
       "4                19014.623047              17206.958984  \n",
       "\n",
       "[5 rows x 57 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count Encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FrequencyEncoder:\n",
    "    \n",
    "    def __init__(self, seed=42, mode='gpu'):\n",
    "        self.seed = seed\n",
    "        if mode=='gpu':\n",
    "            self.np = cupy\n",
    "            self.df = cudf\n",
    "        else:\n",
    "            self.np = np\n",
    "            self.df = pd\n",
    "        self.mode = mode\n",
    "        \n",
    "    def fit_transform(self, train, x_col, c_col=None, out_col = None):\n",
    "        self.np.random.seed(self.seed)\n",
    "        if c_col is None or c_col not in train.columns:\n",
    "            c_col = 'dummy'\n",
    "            train[c_col] = 1\n",
    "            drop = True\n",
    "        else:\n",
    "            drop = False\n",
    "            \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'CE_{tag}_norm'\n",
    "            \n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all = train.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        if drop:\n",
    "            train = train.drop(c_col,axis=1)\n",
    "        agg_all.columns = cols + [out_col]\n",
    "        agg_all[out_col] = agg_all[out_col].astype('int32')\n",
    "        agg_all[out_col] = agg_all[out_col]*1.0/len(train)\n",
    "        agg_all[out_col] = agg_all[out_col].astype('float32')\n",
    "    \n",
    "        train = train.merge(agg_all,on=cols,how='left')\n",
    "        del agg_all\n",
    "        #print(train.columns)\n",
    "        if self.mode=='gpu':\n",
    "            if isinstance(train,dask_cudf.core.DataFrame):\n",
    "                train[out_col] = train.map_partitions(lambda cudf_df: cudf_df[out_col].nans_to_nulls())\n",
    "            else:\n",
    "                train[out_col] = train[out_col].nans_to_nulls()\n",
    "        return train\n",
    "    \n",
    "    def transform(self, test, x_col, c_col=None, out_col = None):\n",
    "        return self.fit_transform(test, x_col, c_col, out_col)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CountEncoder:\n",
    "    \n",
    "    def __init__(self, seed=42, mode='gpu'):\n",
    "        self.seed = seed\n",
    "        if mode=='gpu':\n",
    "            self.np = cupy\n",
    "            self.df = cudf\n",
    "        else:\n",
    "            self.np = np\n",
    "            self.df = pd\n",
    "        self.mode = mode\n",
    "        \n",
    "    def fit_transform(self, train, test, x_col, out_col = None):\n",
    "        self.np.random.seed(self.seed)\n",
    "        \n",
    "        common_cols = [i for i in train.columns if i in test.columns and i!=x_col]\n",
    "\n",
    "        if len(common_cols):\n",
    "            c_col = common_cols[0]\n",
    "            drop = False\n",
    "        else:\n",
    "            c_col = 'dummy'\n",
    "            train[c_col] = 1\n",
    "            test[c_col]=1\n",
    "            drop = True\n",
    "            \n",
    "        if out_col is None:\n",
    "            tag = x_col if isinstance(x_col,str) else '_'.join(x_col)\n",
    "            out_col = f'CE_{tag}_norm'\n",
    "            \n",
    "        cols = [x_col] if isinstance(x_col,str) else x_col\n",
    "        agg_all = train.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        agg_all.columns = cols + [out_col]\n",
    "        \n",
    "        agg_test = test.groupby(cols).agg({c_col:'count'}).reset_index()\n",
    "        agg_test.columns = cols + [out_col+'_test']\n",
    "        agg_all = agg_all.merge(agg_test,on=cols,how='left')\n",
    "        agg_all[out_col+'_test'] = agg_all[out_col+'_test'].fillna(0)\n",
    "        agg_all[out_col] = agg_all[out_col] + agg_all[out_col+'_test']\n",
    "        agg_all = agg_all.drop(out_col+'_test', axis=1)\n",
    "        del agg_test\n",
    "            \n",
    "        if drop:\n",
    "            train = train.drop(c_col,axis=1)\n",
    "            test = test.drop(c_col,axis=1)\n",
    "        train = train.merge(agg_all,on=cols,how='left')\n",
    "        test = test.merge(agg_all,on=cols,how='left')\n",
    "        del agg_all\n",
    "        return train,test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.82 s, sys: 187 ms, total: 2 s\n",
      "Wall time: 7.48 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF CE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "    encoder = CountEncoder()\n",
    "    out_col = f'CE_{c}'\n",
    "    train,valid = encoder.fit_transform(train, valid, c, out_col=out_col)\n",
    "    print\n",
    "    del encoder\n",
    "    train,valid = dask.persist(train,valid)\n",
    "    train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.77 s, sys: 252 ms, total: 3.02 s\n",
      "Wall time: 9.22 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# cuDF CE ENCODING IS SUPER FAST!!\n",
    "idx = 0; cols = []\n",
    "for c in ['media', 'tweet_type', 'language', 'a_user_id', 'b_user_id']:\n",
    "    encoder = FrequencyEncoder()\n",
    "    out_col = f'CE_{c}_norm'\n",
    "    train = encoder.fit_transform(train, c, c_col='tweet_id', out_col=out_col)\n",
    "    valid = encoder.transform(valid, c, c_col='tweet_id', out_col=out_col)\n",
    "    cols.append(out_col)\n",
    "    del encoder\n",
    "    train,valid = dask.persist(train,valid)\n",
    "    train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Difference Encode (Lag Features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_encode_cudf_v1(train,col,tar,sft=1):\n",
    "    train[col+'_sft'] = train[col].shift(sft)\n",
    "    train[tar+'_sft'] = train[tar].shift(sft)\n",
    "    out_col = f'DE_{col}_{tar}_{sft}'\n",
    "    train[out_col] = train[tar]-train[tar+'_sft']\n",
    "    mask = '__MASK__'\n",
    "    train[mask] = train[col] == train[col+'_sft']\n",
    "    train = train.drop([col+'_sft',tar+'_sft'],axis=1)\n",
    "    train[out_col] = train[out_col]*train[mask]\n",
    "    train = train.drop(mask,axis=1)\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DE b_user_id b_follower_count 1 1.1 seconds\n",
      "DE b_user_id b_follower_count -1 1.2 seconds\n",
      "DE b_user_id b_following_count 1 1.2 seconds\n",
      "DE b_user_id b_following_count -1 1.0 seconds\n",
      "DE b_user_id language 1 1.3 seconds\n",
      "DE b_user_id language -1 1.2 seconds\n",
      "CPU times: user 4.32 s, sys: 400 ms, total: 4.72 s\n",
      "Wall time: 7.09 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# cuDF DE ENCODING IS FAST!!\n",
    "idx = 0; cols = []; sc = 'timestamp'\n",
    "for c in ['b_user_id']:\n",
    "    for t in ['b_follower_count','b_following_count','language']:\n",
    "        for s in [1,-1]:\n",
    "            start = time.time()\n",
    "            train = diff_encode_cudf_v1(train, col=c, tar=t, sft=s)\n",
    "            valid = diff_encode_cudf_v1(valid, col=c, tar=t, sft=s)\n",
    "            train,valid = dask.persist(train,valid)\n",
    "            train.head()\n",
    "            end = time.time(); idx += 1\n",
    "            print('DE',c,t,s,'%.1f seconds'%(end-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Diff Language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_lang = train[['a_user_id', 'language', 'tweet_id']].drop_duplicates()\n",
    "valid_lang = valid[['a_user_id', 'language', 'tweet_id']].drop_duplicates()\n",
    "train_lang_count = train_lang.groupby(['a_user_id', 'language']).agg({'tweet_id':'count'}).reset_index()\n",
    "valid_lang_count = valid_lang.groupby(['a_user_id', 'language']).agg({'tweet_id':'count'}).reset_index()\n",
    "train_lang_count,valid_lang_count = dask.persist(train_lang_count,valid_lang_count)\n",
    "train_lang_count.head()\n",
    "del train_lang,valid_lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 90.7 ms, sys: 1.84 ms, total: 92.5 ms\n",
      "Wall time: 301 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>top_language</th>\n",
       "      <th>language_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20499</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20499</td>\n",
       "      <td>38</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20499</td>\n",
       "      <td>44</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20499</td>\n",
       "      <td>47</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20499</td>\n",
       "      <td>54</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a_user_id  top_language  language_count\n",
       "0      20499            18               2\n",
       "1      20499            38               2\n",
       "2      20499            44              37\n",
       "3      20499            47               5\n",
       "4      20499            54              44"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train_lang_count = train_lang_count.merge(valid_lang_count,on=['a_user_id', 'language'],how='left')\n",
    "train_lang_count['tweet_id_y'] = train_lang_count['tweet_id_y'].fillna(0)\n",
    "train_lang_count['tweet_id_x'] = train_lang_count['tweet_id_x'] + train_lang_count['tweet_id_y']\n",
    "train_lang_count = train_lang_count.drop('tweet_id_y',axis=1)\n",
    "train_lang_count.columns = ['a_user_id', 'top_language', 'language_count']\n",
    "train_lang_count, = dask.persist(train_lang_count)\n",
    "train_lang_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 75.1 ms, sys: 8.69 ms, total: 83.8 ms\n",
      "Wall time: 150 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a_user_id</th>\n",
       "      <th>top_language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70964</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70966</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70951</th>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70954</th>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70967</th>\n",
       "      <td>4</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       a_user_id  top_language\n",
       "70964          0             4\n",
       "70966          1            47\n",
       "70951          2            21\n",
       "70954          3            25\n",
       "70967          4            54"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train_lang_count = train_lang_count.sort_values(['a_user_id', 'language_count'])\n",
    "train_lang_count['a_user_shifted'] = train_lang_count['a_user_id'].shift(1)\n",
    "train_lang_count = train_lang_count[train_lang_count['a_user_id']!=train_lang_count['a_user_shifted']]\n",
    "train_lang_count = train_lang_count.drop(['a_user_shifted','language_count'],axis=1)\n",
    "train_lang_count.columns = ['a_user_id','top_language']\n",
    "train_lang_count, = dask.persist(train_lang_count)\n",
    "train_lang_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_language(df,df_lang_count):\n",
    "    df = df.merge(df_lang_count,how='left', left_on='b_user_id', right_on='a_user_id')\n",
    "    df['nan_language'] = df['top_language'].isnull()\n",
    "    df['same_language'] = df['language'] == df['top_language']\n",
    "    df['diff_language'] = df['language'] != df['top_language']\n",
    "    df['same_language'] = df['same_language']*(1-df['nan_language'])\n",
    "    df['diff_language'] = df['diff_language']*(1-df['nan_language'])\n",
    "    df = df.drop('top_language',axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 768 ms, sys: 86.3 ms, total: 854 ms\n",
      "Wall time: 2.2 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>media</th>\n",
       "      <th>domains</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>a_user_id_x</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>DE_b_user_id_b_follower_count_1</th>\n",
       "      <th>DE_b_user_id_b_follower_count_-1</th>\n",
       "      <th>DE_b_user_id_b_following_count_1</th>\n",
       "      <th>DE_b_user_id_b_following_count_-1</th>\n",
       "      <th>DE_b_user_id_language_1</th>\n",
       "      <th>DE_b_user_id_language_-1</th>\n",
       "      <th>a_user_id_y</th>\n",
       "      <th>nan_language</th>\n",
       "      <th>same_language</th>\n",
       "      <th>diff_language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>117178</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>86219</td>\n",
       "      <td>2963357</td>\n",
       "      <td>156</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>117178</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>86219</td>\n",
       "      <td>2963357</td>\n",
       "      <td>156</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>117178</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>86219</td>\n",
       "      <td>2957303</td>\n",
       "      <td>156</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>117178</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>86219</td>\n",
       "      <td>2963357</td>\n",
       "      <td>156</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>742378</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>1.580948e+09</td>\n",
       "      <td>86219</td>\n",
       "      <td>2963357</td>\n",
       "      <td>156</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id  media  domains  tweet_type  language     timestamp  a_user_id_x  \\\n",
       "0    117178      0        0           0        11  1.580948e+09        86219   \n",
       "1    117178      0        0           0        11  1.580948e+09        86219   \n",
       "2    117178      0        0           0        11  1.580948e+09        86219   \n",
       "3    117178      0        0           0        11  1.580948e+09        86219   \n",
       "4    742378      0        0           2        11  1.580948e+09        86219   \n",
       "\n",
       "   a_follower_count  a_following_count  a_is_verified  ...  \\\n",
       "0           2963357                156           True  ...   \n",
       "1           2963357                156           True  ...   \n",
       "2           2957303                156           True  ...   \n",
       "3           2963357                156           True  ...   \n",
       "4           2963357                156           True  ...   \n",
       "\n",
       "  DE_b_user_id_b_follower_count_1  DE_b_user_id_b_follower_count_-1  \\\n",
       "0                               0                                 0   \n",
       "1                               0                                 0   \n",
       "2                               0                                 0   \n",
       "3                               0                                 0   \n",
       "4                               0                                 0   \n",
       "\n",
       "   DE_b_user_id_b_following_count_1  DE_b_user_id_b_following_count_-1  \\\n",
       "0                                 0                                  0   \n",
       "1                                 0                                  0   \n",
       "2                                 0                                  0   \n",
       "3                                 0                                  0   \n",
       "4                                 0                                  0   \n",
       "\n",
       "   DE_b_user_id_language_1  DE_b_user_id_language_-1  a_user_id_y  \\\n",
       "0                        0                         0         null   \n",
       "1                        0                         0         null   \n",
       "2                        0                         0         null   \n",
       "3                        0                         0         null   \n",
       "4                        0                         0         null   \n",
       "\n",
       "  nan_language  same_language  diff_language  \n",
       "0         True              0              0  \n",
       "1         True              0              0  \n",
       "2         True              0              0  \n",
       "3         True              0              0  \n",
       "4         True              0              0  \n",
       "\n",
       "[5 rows x 78 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train = diff_language(train,train_lang_count)\n",
    "valid = diff_language(valid,train_lang_count)\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rate feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 137 ms, sys: 4.33 ms, total: 141 ms\n",
      "Wall time: 133 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# follow rate feature\n",
    "train['a_ff_rate'] = (train['a_following_count'] / train['a_follower_count']).astype('float32')\n",
    "train['b_ff_rate'] = (train['b_follower_count']  / train['b_following_count']).astype('float32')\n",
    "valid['a_ff_rate']  = (valid['a_following_count'] / valid['a_follower_count']).astype('float32')\n",
    "valid['b_ff_rate']  = (valid['b_follower_count']  / valid['b_following_count']).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "train,valid = dask.persist(train,valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train.head()\n",
    "valid.head()\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarize Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.02 ms, sys: 13 µs, total: 4.04 ms\n",
      "Wall time: 3.89 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['timestamp',\n",
       " 'a_account_creation',\n",
       " 'b_account_creation',\n",
       " 'engage_time',\n",
       " 'fold',\n",
       " 'b_user_id',\n",
       " 'dt_dow',\n",
       " 'a_account_creation',\n",
       " 'b_account_creation',\n",
       " 'elapsed_time',\n",
       " 'domains']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "label_names = ['reply', 'retweet', 'retweet_comment', 'like']\n",
    "DONT_USE = ['timestamp','a_account_creation','b_account_creation','engage_time',\n",
    "            'fold','b_user_id','a_user_id', 'dt_dow',\n",
    "            'a_account_creation', 'b_account_creation', 'elapsed_time',\n",
    "             'links','domains','hashtags0','hashtags1']\n",
    "DONT_USE += label_names\n",
    "features = [c for c in train.columns if c not in DONT_USE]\n",
    "\n",
    "RMV = [c for c in DONT_USE if c in train.columns and c not in label_names]\n",
    "RMV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 939 ms, sys: 185 ms, total: 1.12 s\n",
      "Wall time: 1.28 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for col in RMV:\n",
    "    #print(col, col in train.columns)\n",
    "    if col in train.columns:\n",
    "        train = train.drop(col,axis=1)\n",
    "        train, = dask.persist(train)\n",
    "        train.head()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 801 ms, sys: 119 ms, total: 920 ms\n",
      "Wall time: 1.07 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for col in RMV:\n",
    "    #print(col, col in valid.columns)\n",
    "    if col in valid.columns:\n",
    "        valid = valid.drop(col,axis=1)\n",
    "        valid, = dask.persist(valid,)\n",
    "        valid.head()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model Validate\n",
    "We will train on random 10% of first 5 days and validation on last 2 days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107236522\n",
      "10716577\n",
      "Using 66 features: 66\n",
      "CPU times: user 275 ms, sys: 35.2 ms, total: 310 ms\n",
      "Wall time: 588 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['media', 'tweet_type', 'language', 'a_user_id_x',\n",
       "       'a_follower_count', 'a_following_count', 'a_is_verified',\n",
       "       'b_follower_count', 'b_following_count', 'b_is_verified',\n",
       "       'b_follows_a', 'len_domains', 'len_hashtags', 'len_links',\n",
       "       'dt_hour', 'dt_minute', 'dt_second', 'TE_media_reply',\n",
       "       'TE_tweet_type_reply', 'TE_language_reply', 'TE_a_user_id_reply',\n",
       "       'TE_b_user_id_reply', 'TE_media_retweet', 'TE_tweet_type_retweet',\n",
       "       'TE_language_retweet', 'TE_a_user_id_retweet',\n",
       "       'TE_b_user_id_retweet', 'TE_media_retweet_comment',\n",
       "       'TE_tweet_type_retweet_comment', 'TE_language_retweet_comment',\n",
       "       'TE_a_user_id_retweet_comment', 'TE_b_user_id_retweet_comment',\n",
       "       'TE_media_like', 'TE_tweet_type_like', 'TE_language_like',\n",
       "       'TE_a_user_id_like', 'TE_b_user_id_like', 'TE_multi_reply',\n",
       "       'TE_multi_retweet', 'TE_multi_retweet_comment', 'TE_multi_like',\n",
       "       'TE_media_elapsed_time', 'TE_tweet_type_elapsed_time',\n",
       "       'TE_language_elapsed_time', 'CE_media', 'CE_tweet_type',\n",
       "       'CE_language', 'CE_a_user_id', 'CE_b_user_id', 'CE_media_norm',\n",
       "       'CE_tweet_type_norm', 'CE_language_norm', 'CE_a_user_id_norm',\n",
       "       'CE_b_user_id_norm', 'DE_b_user_id_b_follower_count_1',\n",
       "       'DE_b_user_id_b_follower_count_-1',\n",
       "       'DE_b_user_id_b_following_count_1',\n",
       "       'DE_b_user_id_b_following_count_-1', 'DE_b_user_id_language_1',\n",
       "       'DE_b_user_id_language_-1', 'a_user_id_y', 'nan_language',\n",
       "       'same_language', 'diff_language', 'a_ff_rate', 'b_ff_rate'],\n",
       "      dtype='<U33')"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "SAMPLE_RATIO = 0.1\n",
    "SEED = 1\n",
    "\n",
    "if SAMPLE_RATIO < 1.0:\n",
    "    train['sample'] = train['tweet_id'].map_partitions(lambda cudf_df: cudf_df.hash_encode(stop=10))\n",
    "    print(len(train))\n",
    "    \n",
    "    train = train[train['sample']<10*SAMPLE_RATIO]\n",
    "    train, = dask.persist(train)\n",
    "    train.head()\n",
    "    print(len(train))\n",
    "\n",
    "\n",
    "Y_train = train[label_names]\n",
    "Y_train, = dask.persist(Y_train)\n",
    "Y_train.head()    \n",
    "    \n",
    "train = train.drop(['sample','tweet_id']+label_names,axis=1)\n",
    "train, = dask.persist(train)\n",
    "train.head()\n",
    "\n",
    "\n",
    "features = [c for c in train.columns if c not in DONT_USE]\n",
    "print('Using %i features:'%(len(features)),train.shape[1])\n",
    "np.asarray(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40838716\n",
      "16288926\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id_x</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>DE_b_user_id_b_following_count_1</th>\n",
       "      <th>DE_b_user_id_b_following_count_-1</th>\n",
       "      <th>DE_b_user_id_language_1</th>\n",
       "      <th>DE_b_user_id_language_-1</th>\n",
       "      <th>a_user_id_y</th>\n",
       "      <th>nan_language</th>\n",
       "      <th>same_language</th>\n",
       "      <th>diff_language</th>\n",
       "      <th>a_ff_rate</th>\n",
       "      <th>b_ff_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "      <td>140</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>964466</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.071429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>True</td>\n",
       "      <td>513</td>\n",
       "      <td>728</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8936009</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.704670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>True</td>\n",
       "      <td>15</td>\n",
       "      <td>217</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.069124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>True</td>\n",
       "      <td>87</td>\n",
       "      <td>221</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.393665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.042857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    media  tweet_type  language  a_user_id_x  a_follower_count  \\\n",
       "33      5           2        54         1579          34907345   \n",
       "34      5           2        54         1579          34907345   \n",
       "35      5           2        54         1579          34907345   \n",
       "36      5           2        54         1579          34907345   \n",
       "37      5           2        54         1579          34907345   \n",
       "\n",
       "    a_following_count  a_is_verified  b_follower_count  b_following_count  \\\n",
       "33                243           True                10                140   \n",
       "34                243           True               513                728   \n",
       "35                243           True                15                217   \n",
       "36                243           True                87                221   \n",
       "37                243           True                 3                 70   \n",
       "\n",
       "    b_is_verified  ...  DE_b_user_id_b_following_count_1  \\\n",
       "33          False  ...                                 0   \n",
       "34          False  ...                                 0   \n",
       "35          False  ...                                 0   \n",
       "36          False  ...                                 0   \n",
       "37          False  ...                                 0   \n",
       "\n",
       "    DE_b_user_id_b_following_count_-1  DE_b_user_id_language_1  \\\n",
       "33                                  0                        0   \n",
       "34                                  0                        0   \n",
       "35                                  0                        0   \n",
       "36                                  0                        0   \n",
       "37                                  0                        0   \n",
       "\n",
       "    DE_b_user_id_language_-1  a_user_id_y nan_language  same_language  \\\n",
       "33                         0       964466        False              1   \n",
       "34                         0      8936009        False              0   \n",
       "35                         0         null         True              0   \n",
       "36                         0         null         True              0   \n",
       "37                         0         null         True              0   \n",
       "\n",
       "    diff_language  a_ff_rate  b_ff_rate  \n",
       "33              0   0.000007   0.071429  \n",
       "34              1   0.000007   0.704670  \n",
       "35              0   0.000007   0.069124  \n",
       "36              0   0.000007   0.393665  \n",
       "37              0   0.000007   0.042857  \n",
       "\n",
       "[5 rows x 66 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SAMPLE_RATIO = 0.35 # VAL SET NOW SIZE OF TEST SET\n",
    "SEED = 1\n",
    "if SAMPLE_RATIO < 1.0:\n",
    "    print(len(valid))\n",
    "    valid['sample'] = valid['tweet_id'].map_partitions(lambda cudf_df: cudf_df.hash_encode(stop=10))\n",
    "    \n",
    "    valid = valid[valid['sample']<10*SAMPLE_RATIO]\n",
    "    valid, = dask.persist(valid)\n",
    "    valid.head()\n",
    "    print(len(valid))\n",
    "    \n",
    "Y_valid = valid[label_names]\n",
    "Y_valid, = dask.persist(Y_valid)\n",
    "Y_valid.head()\n",
    "\n",
    "valid = valid.drop(['sample','tweet_id']+label_names,axis=1)\n",
    "valid, = dask.persist(valid)\n",
    "valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGB Version 1.2.0-SNAPSHOT\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "print('XGB Version',xgb.__version__)\n",
    "\n",
    "xgb_parms = { \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.1, \n",
    "    'subsample':0.8,\n",
    "    'colsample_bytree':0.3, \n",
    "    'eval_metric':'logloss',\n",
    "    'objective':'binary:logistic',\n",
    "    'tree_method':'gpu_hist',\n",
    "    'predictor' : 'gpu_predictor'\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no dup :) \n",
      "X_train.shape (Delayed('int-a33f43d7-84ea-407c-aa6d-be1f5f7e1b97'), 66)\n",
      "X_valid.shape (Delayed('int-5d45c190-cacd-49c9-9462-07b90197567e'), 66)\n"
     ]
    }
   ],
   "source": [
    "if train.columns.duplicated().sum()>0:\n",
    "    raise Exception(f'duplicated!: { train.columns[train.columns.duplicated()] }')\n",
    "print('no dup :) ')\n",
    "print(f'X_train.shape {train.shape}')\n",
    "print(f'X_valid.shape {valid.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 444 ms, sys: 47.5 ms, total: 492 ms\n",
      "Wall time: 1.62 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>media</th>\n",
       "      <th>tweet_type</th>\n",
       "      <th>language</th>\n",
       "      <th>a_user_id_x</th>\n",
       "      <th>a_follower_count</th>\n",
       "      <th>a_following_count</th>\n",
       "      <th>a_is_verified</th>\n",
       "      <th>b_follower_count</th>\n",
       "      <th>b_following_count</th>\n",
       "      <th>b_is_verified</th>\n",
       "      <th>...</th>\n",
       "      <th>DE_b_user_id_b_following_count_1</th>\n",
       "      <th>DE_b_user_id_b_following_count_-1</th>\n",
       "      <th>DE_b_user_id_language_1</th>\n",
       "      <th>DE_b_user_id_language_-1</th>\n",
       "      <th>a_user_id_y</th>\n",
       "      <th>nan_language</th>\n",
       "      <th>same_language</th>\n",
       "      <th>diff_language</th>\n",
       "      <th>a_ff_rate</th>\n",
       "      <th>b_ff_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>140</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>964466</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.071429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>1</td>\n",
       "      <td>513</td>\n",
       "      <td>728</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8936009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.704670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>217</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.069124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>1</td>\n",
       "      <td>87</td>\n",
       "      <td>221</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.393665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>54</td>\n",
       "      <td>1579</td>\n",
       "      <td>34907345</td>\n",
       "      <td>243</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.042857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    media  tweet_type  language  a_user_id_x  a_follower_count  \\\n",
       "33      5           2        54         1579          34907345   \n",
       "34      5           2        54         1579          34907345   \n",
       "35      5           2        54         1579          34907345   \n",
       "36      5           2        54         1579          34907345   \n",
       "37      5           2        54         1579          34907345   \n",
       "\n",
       "    a_following_count  a_is_verified  b_follower_count  b_following_count  \\\n",
       "33                243              1                10                140   \n",
       "34                243              1               513                728   \n",
       "35                243              1                15                217   \n",
       "36                243              1                87                221   \n",
       "37                243              1                 3                 70   \n",
       "\n",
       "    b_is_verified  ...  DE_b_user_id_b_following_count_1  \\\n",
       "33              0  ...                                 0   \n",
       "34              0  ...                                 0   \n",
       "35              0  ...                                 0   \n",
       "36              0  ...                                 0   \n",
       "37              0  ...                                 0   \n",
       "\n",
       "    DE_b_user_id_b_following_count_-1  DE_b_user_id_language_1  \\\n",
       "33                                  0                        0   \n",
       "34                                  0                        0   \n",
       "35                                  0                        0   \n",
       "36                                  0                        0   \n",
       "37                                  0                        0   \n",
       "\n",
       "    DE_b_user_id_language_-1  a_user_id_y nan_language  same_language  \\\n",
       "33                         0       964466            0              1   \n",
       "34                         0      8936009            0              0   \n",
       "35                         0         null            1              0   \n",
       "36                         0         null            1              0   \n",
       "37                         0         null            1              0   \n",
       "\n",
       "    diff_language  a_ff_rate  b_ff_rate  \n",
       "33              0   0.000007   0.071429  \n",
       "34              1   0.000007   0.704670  \n",
       "35              0   0.000007   0.069124  \n",
       "36              0   0.000007   0.393665  \n",
       "37              0   0.000007   0.042857  \n",
       "\n",
       "[5 rows x 66 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for col in train.columns:\n",
    "    if train[col].dtype=='bool':\n",
    "        train[col] = train[col].astype('int8')\n",
    "        valid[col] = valid[col].astype('int8')\n",
    "train,valid = dask.persist(train,valid)\n",
    "train.head()\n",
    "valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################\n",
      "### reply\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 13.8 seconds\n",
      "Predicting...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jiwei/anaconda3/envs/rapids/lib/python3.7/site-packages/distributed/worker.py:3349: UserWarning: Large object of size 4.57 MB detected in task graph: \n",
      "  [<function predict.<locals>.mapped_predict at 0x7f ... titions>, True]\n",
      "Consider scattering large objects ahead of time\n",
      "with client.scatter to reduce scheduler burden and \n",
      "keep data on workers\n",
      "\n",
      "    future = client.submit(func, big_data)    # bad\n",
      "\n",
      "    big_future = client.scatter(big_data)     # good\n",
      "    future = client.submit(func, big_future)  # good\n",
      "  % (format_bytes(len(b)), s)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took 0.5 seconds\n",
      "#########################\n",
      "### retweet\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 13.5 seconds\n",
      "Predicting...\n",
      "Took 0.6 seconds\n",
      "#########################\n",
      "### retweet_comment\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 12.9 seconds\n",
      "Predicting...\n",
      "Took 0.5 seconds\n",
      "#########################\n",
      "### like\n",
      "#########################\n",
      "Creating DMatrix...\n",
      "Took 0.1 seconds\n",
      "Training...\n",
      "Took 13.6 seconds\n",
      "Predicting...\n",
      "Took 0.6 seconds\n",
      "CPU times: user 4.09 s, sys: 581 ms, total: 4.67 s\n",
      "Wall time: 56.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# TRAIN AND VALIDATE\n",
    "\n",
    "NROUND = 300\n",
    "VERBOSE_EVAL = 50\n",
    "#ESR = 50\n",
    "    \n",
    "oof = np.zeros((len(valid),len(label_names)))\n",
    "preds = []\n",
    "for i in range(4):\n",
    "\n",
    "    name = label_names[i]\n",
    "    print('#'*25);print('###',name);print('#'*25)\n",
    "       \n",
    "    start = time.time(); print('Creating DMatrix...')\n",
    "    dtrain = xgb.dask.DaskDMatrix(client,data=train,label=Y_train.iloc[:, i])\n",
    "    dvalid = xgb.dask.DaskDMatrix(client,data=valid,label=Y_valid.iloc[:, i])\n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "             \n",
    "    start = time.time(); print('Training...')\n",
    "    model = xgb.dask.train(client, xgb_parms, \n",
    "                           dtrain=dtrain,\n",
    "                           #evals=[(dtrain,'train'),(dvalid,'valid')],\n",
    "                           num_boost_round=NROUND,\n",
    "                           #early_stopping_rounds=ESR,\n",
    "                           verbose_eval=VERBOSE_EVAL) \n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "        \n",
    "    start = time.time(); print('Predicting...')\n",
    "    #Y_valid[f'pred_{name}'] = xgb.dask.predict(client,model,valid)\n",
    "    #oof[:, i] += xgb.dask.predict(client,model,dvalid).compute()\n",
    "    preds.append(xgb.dask.predict(client,model,valid))\n",
    "    print('Took %.1f seconds'%(time.time()-start))\n",
    "        \n",
    "    del model, dtrain, dvalid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "for i in preds:\n",
    "    print(i.isnull().sum().compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "yvalid = Y_valid[label_names].values.compute()\n",
    "yvalid = cupy.asnumpy(yvalid)\n",
    "\n",
    "oof = cupy.array([i.values.compute() for i in preds]).T\n",
    "oof = cupy.asnumpy(oof)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute Validation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve, auc, log_loss\n",
    "\n",
    "def compute_prauc(pred, gt):\n",
    "  prec, recall, thresh = precision_recall_curve(gt, pred)\n",
    "  prauc = auc(recall, prec)\n",
    "  return prauc\n",
    "\n",
    "def calculate_ctr(gt):\n",
    "  positive = len([x for x in gt if x == 1])\n",
    "  ctr = positive/float(len(gt))\n",
    "  return ctr\n",
    "\n",
    "def compute_rce(pred, gt):\n",
    "    cross_entropy = log_loss(gt, pred)\n",
    "    data_ctr = calculate_ctr(gt)\n",
    "    strawman_cross_entropy = log_loss(gt, [data_ctr for _ in range(len(gt))])\n",
    "    return (1.0 - cross_entropy/strawman_cross_entropy)*100.0\n",
    "\n",
    "# FAST METRIC FROM GIBA\n",
    "def compute_rce_fast(pred, gt):\n",
    "    cross_entropy = log_loss(gt, pred)\n",
    "    yt = np.mean(gt)     \n",
    "    strawman_cross_entropy = -(yt*np.log(yt) + (1 - yt)*np.log(1 - yt))\n",
    "    return (1.0 - cross_entropy/strawman_cross_entropy)*100.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reply                PRAUC:0.18020 RCE:19.69112\n",
      "retweet              PRAUC:0.53612 RCE:29.32064\n",
      "retweet_comment      PRAUC:0.05623 RCE:11.93086\n",
      "like                 PRAUC:0.78253 RCE:26.83609\n",
      "CPU times: user 36.6 s, sys: 2.58 s, total: 39.1 s\n",
      "Wall time: 37.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "txt = ''\n",
    "for i in range(4):\n",
    "    prauc = compute_prauc(oof[:,i], yvalid[:, i])\n",
    "    rce   = compute_rce_fast(oof[:,i], yvalid[:, i])\n",
    "    txt_ = f\"{label_names[i]:20} PRAUC:{prauc:.5f} RCE:{rce:.5f}\"\n",
    "    print(txt_)\n",
    "    txt += txt_ + '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook took 4.5 minutes\n"
     ]
    }
   ],
   "source": [
    "print('This notebook took %.1f minutes'%((time.time()-very_start)/60.))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
